#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2024/1/30 17:55
# @Author  : 王凯
# @File    : error_collector.py
# @Project : scrapy_spider
from scrapy.downloadermiddlewares.retry import RetryMiddleware


class ErrorCollectorMiddleware(RetryMiddleware):
    def process_exception(self, request, exception, spider):
        ...
        # if spider.crawler.stats:
        #     error_url_mapping = spider.crawler.stats.get_value("error_url_mapping", {})
        #     exception_key = exception.__class__.__name__
        #     error_map_list = error_url_mapping.get(exception_key) or []
        #     error_map_list.append(request.url)
        #     error_map_list = list(set(error_map_list))
        #     error_url_mapping.update({exception_key: error_map_list})
        #     spider.crawler.stats.set_value("error_url_mapping", error_url_mapping)
